<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport"/>
  <title>
   我们是这样理解语言的之 统计语言模型  | 数螺 | NAUT IDEA
  </title>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" rel="stylesheet"/>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"/>
  <style type="text/css">
   #xmain img {
                  max-width: 100%;
                  display: block;
                  margin-top: 10px;
                  margin-bottom: 10px;
                }

                #xmain p {
                    line-height:150%;
                    font-size: 16px;
                    margin-top: 20px;
                }

                #xmain h2 {
                    font-size: 24px;
                }

                #xmain h3 {
                    font-size: 20px;
                }

                #xmain h4 {
                    font-size: 18px;
                }


                .header {
	           background-color: #0099ff;
	           color: #ffffff;
	           margin-bottom: 20px;
	        }

	        .header p {
                  margin: 0px;
                  padding: 10px 0;
                  display: inline-block;  
                  vertical-align: middle;
                  font-size: 16px;
               }

               .header a {
                 color: white;
               }

              .header img {
                 height: 25px;
              }
  </style>
  <script src="http://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">
  </script>
  <script src="http://nautstatic-10007657.file.myqcloud.com/static/css/readability.min.js" type="text/javascript">
  </script>
  <script type="text/javascript">
   $(document).ready(function() {
                 var loc = document.location;
                 var uri = {
                  spec: "http://dataunion.org/11266.html",
                  host: "http://dataunion.org",
                  prePath: "http://dataunion.org",
                  scheme: "http",
                  pathBase: "http://dataunion.org/"
                 };
    
                 var documentClone = document.cloneNode(true);
                 var article = new Readability(uri, documentClone).parse();
     
                 document.getElementById("xmain").innerHTML = article.content;
                });
  </script>
  <!-- 1466454849: Accept with keywords: (title(0.25):模型,社区,语言,数盟, topn(0.533333333333):输入法,概率,语言,自然语言处理,数盟,信息,统计学,神经网络,社区,语料,实际,模型,技术,子集,计算,文章,方法,平滑,效果,公式,语料库,稀疏,问题,算法,哈工大,思想,行业资讯,数据,规模,频率).-->
 </head>
 <body onload="">
  <div class="header">
   <div class="container">
    <div class="row">
     <div class="col-xs-6 col-sm-6 text-left">
      <a href="/databee">
       <img src="http://nautidea-10007657.cos.myqcloud.com/logo_white.png"/>
      </a>
      <a href="/databee">
       <p>
        数螺
       </p>
      </a>
     </div>
     <div class="hidden-xs col-sm-6 text-right">
      <p>
       致力于数据科学的推广和知识传播
      </p>
     </div>
    </div>
   </div>
  </div>
  <div class="container text-center">
   <h1>
    我们是这样理解语言的之 统计语言模型
   </h1>
  </div>
  <div class="container" id="xmain">
   ﻿﻿
   <title>
    我们是这样理解语言的之 统计语言模型 | 数盟社区
   </title>
   <!-- All in One SEO Pack 2.2.7.6.2 by Michael Torbert of Semper Fi Web Design[32,65] -->
   <!-- /all in one seo pack -->
   <!--
<div align="center">
<a href="http://strata.oreilly.com.cn/hadoop-big-data-cn?cmp=mp-data-confreg-home-stcn16_dataunion_pc" target="_blank"><img src="http://dataunion.org/wp-content/uploads/2016/05/stratabj.jpg"/ ></a>
</div>
-->
   <header id="header-web">
    <div class="header-main">
     <hgroup class="logo">
      <h1>
       <a href="http://dataunion.org/" rel="home" title="数盟社区">
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/logo.png"/>
       </a>
      </h1>
     </hgroup>
     <!--logo-->
     <nav class="header-nav">
      <ul class="menu" id="menu-%e4%b8%bb%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-71" id="menu-item-71">
        <a href="http://dataunion.org/category/events" title="events">
         活动
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22457" id="menu-item-22457">
          <a href="http://dataunion.org/2016timeline">
           2016档期
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22459" id="menu-item-22459">
          <a href="http://dataunion.org/category/parterc">
           合作会议
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-has-children menu-item-20869" id="menu-item-20869">
        <a href="http://dataunion.org/category/tech" title="articles">
         文章
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20867" id="menu-item-20867">
          <a href="http://dataunion.org/category/tech/base" title="base">
           基础架构
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3302" id="menu-item-3302">
          <a href="http://dataunion.org/category/tech/ai" title="ai">
           人工智能
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3303" id="menu-item-3303">
          <a href="http://dataunion.org/category/tech/analysis" title="analysis">
           数据分析
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21920" id="menu-item-21920">
          <a href="http://dataunion.org/category/tech/dm">
           数据挖掘
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3314" id="menu-item-3314">
          <a href="http://dataunion.org/category/tech/viz" title="viz">
           可视化
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3305" id="menu-item-3305">
          <a href="http://dataunion.org/category/tech/devl" title="devl">
           编程语言
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-20876" id="menu-item-20876">
        <a href="http://dataunion.org/category/industry">
         行业
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-16328" id="menu-item-16328">
          <a href="http://dataunion.org/category/industry/case" title="case">
           行业应用
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-2112" id="menu-item-2112">
          <a href="http://dataunion.org/category/industry/demo" title="demo">
           Demo展示
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21562" id="menu-item-21562">
          <a href="http://dataunion.org/category/industry/news">
           行业资讯
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-311" id="menu-item-311">
        <a href="http://dataunion.org/category/sources" title="sources">
         资源
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20870" id="menu-item-20870">
        <a href="http://dataunion.org/category/books" title="book">
         图书
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21363" id="menu-item-21363">
        <a href="http://dataunion.org/category/training">
         课程
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-21853" id="menu-item-21853">
        <a href="http://dataunion.org/category/jobs">
         职位
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22050" id="menu-item-22050">
          <a href="http://dataunion.org/category/career">
           职业规划
          </a>
         </li>
        </ul>
       </li>
      </ul>
     </nav>
     <!--header-nav-->
    </div>
   </header>
   <!--header-web-->
   <div id="main">
    <div id="soutab">
     <form action="http://dataunion.org/" class="search" method="get">
     </form>
    </div>
    <div id="container">
     <nav id="mbx">
      当前位置：
      <a href="http://dataunion.org">
       首页
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech">
       文章
      </a>
      &gt;  正文
     </nav>
     <!--mbx-->
     <article class="content">
      <header align="centre" class="contenttitle">
       <div class="mscc">
        <h1 class="mscctitle">
         <a href="http://dataunion.org/11266.html">
          我们是这样理解语言的之 统计语言模型
         </a>
        </h1>
        <address class="msccaddress ">
         <em>
          2,192 次阅读 -
         </em>
         <a href="http://dataunion.org/category/tech" rel="category tag">
          文章
         </a>
        </address>
       </div>
      </header>
      <div class="content-text">
       <p>
        作者：
        <a class="url fn n" href="http://www.flickering.cn/author/fandywang/" rel="author" title="查看所有由fandywang发布的文章">
         fandywang
        </a>
       </p>
       <p>
        记得最早学习
        <a href="http://en.wikipedia.org/wiki/Language_model">
         语言模型
        </a>
        是在研究生的《
        <a href="http://wenku.baidu.com/view/593e9f7e5acfa1c7aa00cc77.html">
         统计自然语言处理
        </a>
        》课上，由哈工大关毅老师主讲，从噪声信道模型切入，到 N-Gram 语言模型的构建、平滑、评价（KL 距离/相对熵、交叉熵、困惑度），接着以音字转换系统（即拼音输入法）为应用实践，最终还引出 Markov 模型和最大熵模型。
       </p>
       <p>
        后来又接触到前腾讯副总裁，现 Goolge 研究员吴军撰写的《
        <a href="http://book.douban.com/subject/10750155/">
         数学之美
        </a>
        》，
        <a href="http://blog.csdn.net/mspinyin/article/category/765944">
         微软拼音输入法团队博客
        </a>
        （微软拼音输入法最初由哈工大王晓龙、关毅等老师完成），逐渐领悟到语言模型，尤其是统计语言模型在众多领域中发挥着重要作用，如下：
       </p>
       <ul>
        <li>
         <strong>
          <strong>
           中文分词：
          </strong>
         </strong>
         P(已/ 结婚/ 的/ 和/ 尚未/ 结婚/ 的/ 青年 | 已结婚的和尚未结婚的青年) &gt; P(已/ 结婚/ 的/ 和尚/ 未/ 结婚/ 的/ 青年 | 已结婚的和尚未结婚的青年)
        </li>
        <li>
         <strong>
          机器翻译：
         </strong>
         P(
         <strong>
          high
         </strong>
         winds tonight | 今晚有大风) &gt; P(
         <strong>
          large
         </strong>
         winds tonight | 今晚有大风)
        </li>
        <li>
         <strong>
          拼写纠错/Query 改写
         </strong>
         ：P(about fifteen
         <strong>
          minutes
         </strong>
         from) &gt; P(about fifteen
         <strong>
          minuets
         </strong>
         from)
        </li>
        <li>
         <strong>
          语音识别
         </strong>
         ：P(I saw a van) &gt;&gt; P(eyes awe of an)
        </li>
        <li>
         <strong>
          音字转换
         </strong>
         ：P(
         <b>
          机器
         </b>
         翻译
         <b>
          及其
         </b>
         应用
         <b>
          激起
         </b>
         了人们
         <b>
          极其
         </b>
         浓厚的学习兴趣 | ji qi fan yi ji qi ying yong ji qi le ren men ji qi nong hou de xue xi xing qu) &gt; P(
         <b>
          机器
         </b>
         翻译
         <b>
          机器
         </b>
         应用
         <b>
          激起
         </b>
         了人们
         <b>
          极其
         </b>
         浓厚的学习兴趣 | ji qi fan yi ji qi ying yong ji qi le ren men ji qi nong hou de xue xi xing qu)
        </li>
        <li>
         <strong>
          自动文摘
         </strong>
         <strong>
          、问答系统
         </strong>
         、
         <strong>
          OCR、… …
         </strong>
        </li>
       </ul>
       <p>
        2011 年刚加入腾讯时，完成的第一个项目是在线广告系统中的 Keyword Extraction，曾尝试对 query log、bidterm、广告创意等建立统计语言模型，计算 Candidate Keyword 的生成概率，作为排序阶段的重要特征之一。
       </p>
       <p>
        接下来，我们不妨一起看下语言模型，尤其是 N-Gram 语言模型的数学表示、效果评估、模型平滑及模型训练（尤其是分布式），为我们后续介绍神经网络语言模型和中文分词技术打个好基础。
       </p>
       <h4>
        语言模型介绍
       </h4>
       <p>
        以上语言模型的各种应用场景可以形式化统一表示如下：
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-1-Frame">
         <span class="math" id="MathJax-Span-1">
          <span class="mrow" id="MathJax-Span-2">
           <span class="mi" id="MathJax-Span-3">
            p
           </span>
           <span class="mo" id="MathJax-Span-4">
            (
           </span>
           <span class="mi" id="MathJax-Span-5">
            S
           </span>
           <span class="mo" id="MathJax-Span-6">
            )
           </span>
           <span class="mo" id="MathJax-Span-7">
            =
           </span>
           <span class="mi" id="MathJax-Span-8">
            p
           </span>
           <span class="mo" id="MathJax-Span-9">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-10">
            <span class="mi" id="MathJax-Span-11">
             w
            </span>
            <span class="mn" id="MathJax-Span-12">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-13">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-14">
            <span class="mi" id="MathJax-Span-15">
             w
            </span>
            <span class="mn" id="MathJax-Span-16">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-17">
            ,
           </span>
           <span class="mo" id="MathJax-Span-18">
            …
           </span>
           <span class="mo" id="MathJax-Span-19">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-20">
            <span class="mi" id="MathJax-Span-21">
             w
            </span>
            <span class="mi" id="MathJax-Span-22">
             n
            </span>
           </span>
           <span class="mo" id="MathJax-Span-23">
            )
           </span>
           <span class="mo" id="MathJax-Span-24">
            =
           </span>
           <span class="mi" id="MathJax-Span-25">
            p
           </span>
           <span class="mi" id="MathJax-Span-26">
            r
           </span>
           <span class="mi" id="MathJax-Span-27">
            o
           </span>
           <span class="msubsup" id="MathJax-Span-28">
            <span class="mi" id="MathJax-Span-29">
             d
            </span>
            <span class="texatom" id="MathJax-Span-30">
             <span class="mrow" id="MathJax-Span-31">
              <span class="mi" id="MathJax-Span-32">
               n
              </span>
             </span>
            </span>
            <span class="texatom" id="MathJax-Span-33">
             <span class="mrow" id="MathJax-Span-34">
              <span class="mi" id="MathJax-Span-35">
               i
              </span>
              <span class="mo" id="MathJax-Span-36">
               =
              </span>
              <span class="mn" id="MathJax-Span-37">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mi" id="MathJax-Span-38">
            p
           </span>
           <span class="mo" id="MathJax-Span-39">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-40">
            <span class="mi" id="MathJax-Span-41">
             w
            </span>
            <span class="mi" id="MathJax-Span-42">
             i
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-43">
            <span class="mrow" id="MathJax-Span-44">
             <span class="mo" id="MathJax-Span-45">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-46">
            <span class="mi" id="MathJax-Span-47">
             w
            </span>
            <span class="mn" id="MathJax-Span-48">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-49">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-50">
            <span class="mi" id="MathJax-Span-51">
             w
            </span>
            <span class="mn" id="MathJax-Span-52">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-53">
            ,
           </span>
           <span class="mo" id="MathJax-Span-54">
            …
           </span>
           <span class="mo" id="MathJax-Span-55">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-56">
            <span class="mi" id="MathJax-Span-57">
             w
            </span>
            <span class="texatom" id="MathJax-Span-58">
             <span class="mrow" id="MathJax-Span-59">
              <span class="mi" id="MathJax-Span-60">
               i
              </span>
              <span class="mo" id="MathJax-Span-61">
               −
              </span>
              <span class="mn" id="MathJax-Span-62">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-63">
            )
           </span>
          </span>
         </span>
        </span>
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-2-Frame">
         <span class="math" id="MathJax-Span-64">
          <span class="mrow" id="MathJax-Span-65">
           <span class="mi" id="MathJax-Span-66">
            p
           </span>
           <span class="mo" id="MathJax-Span-67">
            (
           </span>
           <span class="mi" id="MathJax-Span-68">
            S
           </span>
           <span class="mo" id="MathJax-Span-69">
            )
           </span>
          </span>
         </span>
        </span>
        就是语言模型，即用来计算一个句子
        <span class="MathJax" id="MathJax-Element-3-Frame">
         <span class="math" id="MathJax-Span-70">
          <span class="mrow" id="MathJax-Span-71">
           <span class="mi" id="MathJax-Span-72">
            S
           </span>
          </span>
         </span>
        </span>
        概率的模型。
       </p>
       <p>
        那么，如何计算
        <span class="MathJax" id="MathJax-Element-4-Frame">
         <span class="math" id="MathJax-Span-73">
          <span class="mrow" id="MathJax-Span-74">
           <span class="mi" id="MathJax-Span-75">
            p
           </span>
           <span class="mo" id="MathJax-Span-76">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-77">
            <span class="mi" id="MathJax-Span-78">
             w
            </span>
            <span class="mi" id="MathJax-Span-79">
             i
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-80">
            <span class="mrow" id="MathJax-Span-81">
             <span class="mo" id="MathJax-Span-82">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-83">
            <span class="mi" id="MathJax-Span-84">
             w
            </span>
            <span class="mn" id="MathJax-Span-85">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-86">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-87">
            <span class="mi" id="MathJax-Span-88">
             w
            </span>
            <span class="mn" id="MathJax-Span-89">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-90">
            ,
           </span>
           <span class="mo" id="MathJax-Span-91">
            …
           </span>
           <span class="mo" id="MathJax-Span-92">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-93">
            <span class="mi" id="MathJax-Span-94">
             w
            </span>
            <span class="texatom" id="MathJax-Span-95">
             <span class="mrow" id="MathJax-Span-96">
              <span class="mi" id="MathJax-Span-97">
               i
              </span>
              <span class="mo" id="MathJax-Span-98">
               −
              </span>
              <span class="mn" id="MathJax-Span-99">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-100">
            )
           </span>
          </span>
         </span>
        </span>
        呢？最简单、直接的方法是计数后做除法，即最大似然估计（Maximum Likelihood Estimate，MLE），如下：
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-5-Frame">
         <span class="math" id="MathJax-Span-101">
          <span class="mrow" id="MathJax-Span-102">
           <span class="mi" id="MathJax-Span-103">
            p
           </span>
           <span class="mo" id="MathJax-Span-104">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-105">
            <span class="mi" id="MathJax-Span-106">
             w
            </span>
            <span class="mi" id="MathJax-Span-107">
             i
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-108">
            <span class="mrow" id="MathJax-Span-109">
             <span class="mo" id="MathJax-Span-110">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-111">
            <span class="mi" id="MathJax-Span-112">
             w
            </span>
            <span class="mn" id="MathJax-Span-113">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-114">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-115">
            <span class="mi" id="MathJax-Span-116">
             w
            </span>
            <span class="mn" id="MathJax-Span-117">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-118">
            ,
           </span>
           <span class="mo" id="MathJax-Span-119">
            …
           </span>
           <span class="mo" id="MathJax-Span-120">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-121">
            <span class="mi" id="MathJax-Span-122">
             w
            </span>
            <span class="texatom" id="MathJax-Span-123">
             <span class="mrow" id="MathJax-Span-124">
              <span class="mi" id="MathJax-Span-125">
               i
              </span>
              <span class="mo" id="MathJax-Span-126">
               −
              </span>
              <span class="mn" id="MathJax-Span-127">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-128">
            )
           </span>
           <span class="mo" id="MathJax-Span-129">
            =
           </span>
           <span class="mi" id="MathJax-Span-130">
            f
           </span>
           <span class="mi" id="MathJax-Span-131">
            r
           </span>
           <span class="mi" id="MathJax-Span-132">
            a
           </span>
           <span class="mi" id="MathJax-Span-133">
            c
           </span>
           <span class="texatom" id="MathJax-Span-134">
            <span class="mrow" id="MathJax-Span-135">
             <span class="mi" id="MathJax-Span-136">
              c
             </span>
             <span class="mi" id="MathJax-Span-137">
              o
             </span>
             <span class="mi" id="MathJax-Span-138">
              u
             </span>
             <span class="mi" id="MathJax-Span-139">
              n
             </span>
             <span class="mi" id="MathJax-Span-140">
              t
             </span>
             <span class="mo" id="MathJax-Span-141">
              (
             </span>
             <span class="msubsup" id="MathJax-Span-142">
              <span class="mi" id="MathJax-Span-143">
               w
              </span>
              <span class="mn" id="MathJax-Span-144">
               1
              </span>
             </span>
             <span class="mo" id="MathJax-Span-145">
              ,
             </span>
             <span class="msubsup" id="MathJax-Span-146">
              <span class="mi" id="MathJax-Span-147">
               w
              </span>
              <span class="mn" id="MathJax-Span-148">
               2
              </span>
             </span>
             <span class="mo" id="MathJax-Span-149">
              ,
             </span>
             <span class="mo" id="MathJax-Span-150">
              …
             </span>
             <span class="mo" id="MathJax-Span-151">
              ,
             </span>
             <span class="msubsup" id="MathJax-Span-152">
              <span class="mi" id="MathJax-Span-153">
               w
              </span>
              <span class="texatom" id="MathJax-Span-154">
               <span class="mrow" id="MathJax-Span-155">
                <span class="mi" id="MathJax-Span-156">
                 i
                </span>
                <span class="mo" id="MathJax-Span-157">
                 −
                </span>
                <span class="mn" id="MathJax-Span-158">
                 1
                </span>
               </span>
              </span>
             </span>
             <span class="mo" id="MathJax-Span-159">
              ,
             </span>
             <span class="msubsup" id="MathJax-Span-160">
              <span class="mi" id="MathJax-Span-161">
               w
              </span>
              <span class="mi" id="MathJax-Span-162">
               i
              </span>
             </span>
             <span class="mo" id="MathJax-Span-163">
              )
             </span>
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-164">
            <span class="mrow" id="MathJax-Span-165">
             <span class="mi" id="MathJax-Span-166">
              c
             </span>
             <span class="mi" id="MathJax-Span-167">
              o
             </span>
             <span class="mi" id="MathJax-Span-168">
              u
             </span>
             <span class="mi" id="MathJax-Span-169">
              n
             </span>
             <span class="mi" id="MathJax-Span-170">
              t
             </span>
             <span class="mo" id="MathJax-Span-171">
              (
             </span>
             <span class="msubsup" id="MathJax-Span-172">
              <span class="mi" id="MathJax-Span-173">
               w
              </span>
              <span class="mn" id="MathJax-Span-174">
               1
              </span>
             </span>
             <span class="mo" id="MathJax-Span-175">
              ,
             </span>
             <span class="msubsup" id="MathJax-Span-176">
              <span class="mi" id="MathJax-Span-177">
               w
              </span>
              <span class="mn" id="MathJax-Span-178">
               2
              </span>
             </span>
             <span class="mo" id="MathJax-Span-179">
              ,
             </span>
             <span class="mo" id="MathJax-Span-180">
              …
             </span>
             <span class="mo" id="MathJax-Span-181">
              ,
             </span>
             <span class="msubsup" id="MathJax-Span-182">
              <span class="mi" id="MathJax-Span-183">
               w
              </span>
              <span class="texatom" id="MathJax-Span-184">
               <span class="mrow" id="MathJax-Span-185">
                <span class="mi" id="MathJax-Span-186">
                 i
                </span>
                <span class="mo" id="MathJax-Span-187">
                 −
                </span>
                <span class="mn" id="MathJax-Span-188">
                 1
                </span>
               </span>
              </span>
             </span>
             <span class="mo" id="MathJax-Span-189">
              )
             </span>
            </span>
           </span>
          </span>
         </span>
        </span>
       </p>
       <p>
        其中，
        <span class="MathJax" id="MathJax-Element-6-Frame">
         <span class="math" id="MathJax-Span-190">
          <span class="mrow" id="MathJax-Span-191">
           <span class="mi" id="MathJax-Span-192">
            c
           </span>
           <span class="mi" id="MathJax-Span-193">
            o
           </span>
           <span class="mi" id="MathJax-Span-194">
            u
           </span>
           <span class="mi" id="MathJax-Span-195">
            n
           </span>
           <span class="mi" id="MathJax-Span-196">
            t
           </span>
           <span class="mo" id="MathJax-Span-197">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-198">
            <span class="mi" id="MathJax-Span-199">
             w
            </span>
            <span class="mn" id="MathJax-Span-200">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-201">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-202">
            <span class="mi" id="MathJax-Span-203">
             w
            </span>
            <span class="mn" id="MathJax-Span-204">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-205">
            ,
           </span>
           <span class="mo" id="MathJax-Span-206">
            …
           </span>
           <span class="mo" id="MathJax-Span-207">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-208">
            <span class="mi" id="MathJax-Span-209">
             w
            </span>
            <span class="texatom" id="MathJax-Span-210">
             <span class="mrow" id="MathJax-Span-211">
              <span class="mi" id="MathJax-Span-212">
               i
              </span>
              <span class="mo" id="MathJax-Span-213">
               −
              </span>
              <span class="mn" id="MathJax-Span-214">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-215">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-216">
            <span class="mi" id="MathJax-Span-217">
             w
            </span>
            <span class="mi" id="MathJax-Span-218">
             i
            </span>
           </span>
           <span class="mo" id="MathJax-Span-219">
            )
           </span>
          </span>
         </span>
        </span>
        表示词序列
        <span class="MathJax" id="MathJax-Element-7-Frame">
         <span class="math" id="MathJax-Span-220">
          <span class="mrow" id="MathJax-Span-221">
           <span class="mo" id="MathJax-Span-222">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-223">
            <span class="mi" id="MathJax-Span-224">
             w
            </span>
            <span class="mn" id="MathJax-Span-225">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-226">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-227">
            <span class="mi" id="MathJax-Span-228">
             w
            </span>
            <span class="mn" id="MathJax-Span-229">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-230">
            ,
           </span>
           <span class="mo" id="MathJax-Span-231">
            …
           </span>
           <span class="mo" id="MathJax-Span-232">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-233">
            <span class="mi" id="MathJax-Span-234">
             w
            </span>
            <span class="texatom" id="MathJax-Span-235">
             <span class="mrow" id="MathJax-Span-236">
              <span class="mi" id="MathJax-Span-237">
               i
              </span>
              <span class="mo" id="MathJax-Span-238">
               −
              </span>
              <span class="mn" id="MathJax-Span-239">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-240">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-241">
            <span class="mi" id="MathJax-Span-242">
             w
            </span>
            <span class="mi" id="MathJax-Span-243">
             i
            </span>
           </span>
           <span class="mo" id="MathJax-Span-244">
            )
           </span>
          </span>
         </span>
        </span>
        在语料库中出现的频率。这里面临两个重要的问题：数据稀疏严重和参数空间过大，导致无法实用。
       </p>
       <p>
        实际中，我们一般较常使用的是 N 元语法模型（N-Gram），它采用了马尔科夫假设（Markov Assumption），即认为语言中每个单词只与其前面长度 N-1 的上下文有关。
       </p>
       <ul>
        <li>
         假设下一个词的出现依赖它前面的一个词，即 bigram，则有：
        </li>
       </ul>
       <p>
        <span class="MathJax" id="MathJax-Element-8-Frame">
         <span class="math" id="MathJax-Span-245">
          <span class="mrow" id="MathJax-Span-246">
           <span class="mi" id="MathJax-Span-247">
            p
           </span>
           <span class="mo" id="MathJax-Span-248">
            (
           </span>
           <span class="mi" id="MathJax-Span-249">
            S
           </span>
           <span class="mo" id="MathJax-Span-250">
            )
           </span>
           <span class="mo" id="MathJax-Span-251">
            =
           </span>
           <span class="mi" id="MathJax-Span-252">
            p
           </span>
           <span class="mo" id="MathJax-Span-253">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-254">
            <span class="mi" id="MathJax-Span-255">
             w
            </span>
            <span class="mn" id="MathJax-Span-256">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-257">
            )
           </span>
           <span class="mi" id="MathJax-Span-258">
            p
           </span>
           <span class="mo" id="MathJax-Span-259">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-260">
            <span class="mi" id="MathJax-Span-261">
             w
            </span>
            <span class="mn" id="MathJax-Span-262">
             2
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-263">
            <span class="mrow" id="MathJax-Span-264">
             <span class="mo" id="MathJax-Span-265">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-266">
            <span class="mi" id="MathJax-Span-267">
             w
            </span>
            <span class="mn" id="MathJax-Span-268">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-269">
            )
           </span>
           <span class="mi" id="MathJax-Span-270">
            p
           </span>
           <span class="mo" id="MathJax-Span-271">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-272">
            <span class="mi" id="MathJax-Span-273">
             w
            </span>
            <span class="mn" id="MathJax-Span-274">
             3
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-275">
            <span class="mrow" id="MathJax-Span-276">
             <span class="mo" id="MathJax-Span-277">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-278">
            <span class="mi" id="MathJax-Span-279">
             w
            </span>
            <span class="mn" id="MathJax-Span-280">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-281">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-282">
            <span class="mi" id="MathJax-Span-283">
             w
            </span>
            <span class="mn" id="MathJax-Span-284">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-285">
            )
           </span>
           <span class="mo" id="MathJax-Span-286">
            …
           </span>
           <span class="mi" id="MathJax-Span-287">
            p
           </span>
           <span class="mo" id="MathJax-Span-288">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-289">
            <span class="mi" id="MathJax-Span-290">
             w
            </span>
            <span class="mi" id="MathJax-Span-291">
             n
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-292">
            <span class="mrow" id="MathJax-Span-293">
             <span class="mo" id="MathJax-Span-294">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-295">
            <span class="mi" id="MathJax-Span-296">
             w
            </span>
            <span class="mn" id="MathJax-Span-297">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-298">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-299">
            <span class="mi" id="MathJax-Span-300">
             w
            </span>
            <span class="mn" id="MathJax-Span-301">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-302">
            ,
           </span>
           <span class="mo" id="MathJax-Span-303">
            …
           </span>
           <span class="mo" id="MathJax-Span-304">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-305">
            <span class="mi" id="MathJax-Span-306">
             w
            </span>
            <span class="texatom" id="MathJax-Span-307">
             <span class="mrow" id="MathJax-Span-308">
              <span class="mi" id="MathJax-Span-309">
               n
              </span>
              <span class="mo" id="MathJax-Span-310">
               −
              </span>
              <span class="mn" id="MathJax-Span-311">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-312">
            )
           </span>
           <span class="mo" id="MathJax-Span-313">
            =
           </span>
           <span class="mi" id="MathJax-Span-314">
            p
           </span>
           <span class="mo" id="MathJax-Span-315">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-316">
            <span class="mi" id="MathJax-Span-317">
             w
            </span>
            <span class="mn" id="MathJax-Span-318">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-319">
            )
           </span>
           <span class="mi" id="MathJax-Span-320">
            p
           </span>
           <span class="mo" id="MathJax-Span-321">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-322">
            <span class="mi" id="MathJax-Span-323">
             w
            </span>
            <span class="mn" id="MathJax-Span-324">
             2
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-325">
            <span class="mrow" id="MathJax-Span-326">
             <span class="mo" id="MathJax-Span-327">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-328">
            <span class="mi" id="MathJax-Span-329">
             w
            </span>
            <span class="mn" id="MathJax-Span-330">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-331">
            )
           </span>
           <span class="mi" id="MathJax-Span-332">
            p
           </span>
           <span class="mo" id="MathJax-Span-333">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-334">
            <span class="mi" id="MathJax-Span-335">
             w
            </span>
            <span class="mn" id="MathJax-Span-336">
             3
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-337">
            <span class="mrow" id="MathJax-Span-338">
             <span class="mo" id="MathJax-Span-339">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-340">
            <span class="mi" id="MathJax-Span-341">
             w
            </span>
            <span class="mn" id="MathJax-Span-342">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-343">
            )
           </span>
           <span class="mo" id="MathJax-Span-344">
            …
           </span>
           <span class="mi" id="MathJax-Span-345">
            p
           </span>
           <span class="mo" id="MathJax-Span-346">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-347">
            <span class="mi" id="MathJax-Span-348">
             w
            </span>
            <span class="mi" id="MathJax-Span-349">
             n
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-350">
            <span class="mrow" id="MathJax-Span-351">
             <span class="mo" id="MathJax-Span-352">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-353">
            <span class="mi" id="MathJax-Span-354">
             w
            </span>
            <span class="texatom" id="MathJax-Span-355">
             <span class="mrow" id="MathJax-Span-356">
              <span class="mi" id="MathJax-Span-357">
               n
              </span>
              <span class="mo" id="MathJax-Span-358">
               −
              </span>
              <span class="mn" id="MathJax-Span-359">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-360">
            )
           </span>
          </span>
         </span>
        </span>
       </p>
       <ul>
        <li>
         假设下一个词的出现依赖它前面的两个词，即 trigram，则有：
        </li>
       </ul>
       <p>
        <span class="MathJax" id="MathJax-Element-9-Frame">
         <span class="math" id="MathJax-Span-361">
          <span class="mrow" id="MathJax-Span-362">
           <span class="mi" id="MathJax-Span-363">
            p
           </span>
           <span class="mo" id="MathJax-Span-364">
            (
           </span>
           <span class="mi" id="MathJax-Span-365">
            S
           </span>
           <span class="mo" id="MathJax-Span-366">
            )
           </span>
           <span class="mo" id="MathJax-Span-367">
            =
           </span>
           <span class="mi" id="MathJax-Span-368">
            p
           </span>
           <span class="mo" id="MathJax-Span-369">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-370">
            <span class="mi" id="MathJax-Span-371">
             w
            </span>
            <span class="mn" id="MathJax-Span-372">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-373">
            )
           </span>
           <span class="mi" id="MathJax-Span-374">
            p
           </span>
           <span class="mo" id="MathJax-Span-375">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-376">
            <span class="mi" id="MathJax-Span-377">
             w
            </span>
            <span class="mn" id="MathJax-Span-378">
             2
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-379">
            <span class="mrow" id="MathJax-Span-380">
             <span class="mo" id="MathJax-Span-381">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-382">
            <span class="mi" id="MathJax-Span-383">
             w
            </span>
            <span class="mn" id="MathJax-Span-384">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-385">
            )
           </span>
           <span class="mi" id="MathJax-Span-386">
            p
           </span>
           <span class="mo" id="MathJax-Span-387">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-388">
            <span class="mi" id="MathJax-Span-389">
             w
            </span>
            <span class="mn" id="MathJax-Span-390">
             3
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-391">
            <span class="mrow" id="MathJax-Span-392">
             <span class="mo" id="MathJax-Span-393">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-394">
            <span class="mi" id="MathJax-Span-395">
             w
            </span>
            <span class="mn" id="MathJax-Span-396">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-397">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-398">
            <span class="mi" id="MathJax-Span-399">
             w
            </span>
            <span class="mn" id="MathJax-Span-400">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-401">
            )
           </span>
           <span class="mo" id="MathJax-Span-402">
            …
           </span>
           <span class="mi" id="MathJax-Span-403">
            p
           </span>
           <span class="mo" id="MathJax-Span-404">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-405">
            <span class="mi" id="MathJax-Span-406">
             w
            </span>
            <span class="mi" id="MathJax-Span-407">
             n
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-408">
            <span class="mrow" id="MathJax-Span-409">
             <span class="mo" id="MathJax-Span-410">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-411">
            <span class="mi" id="MathJax-Span-412">
             w
            </span>
            <span class="mn" id="MathJax-Span-413">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-414">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-415">
            <span class="mi" id="MathJax-Span-416">
             w
            </span>
            <span class="mn" id="MathJax-Span-417">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-418">
            ,
           </span>
           <span class="mo" id="MathJax-Span-419">
            …
           </span>
           <span class="mo" id="MathJax-Span-420">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-421">
            <span class="mi" id="MathJax-Span-422">
             w
            </span>
            <span class="texatom" id="MathJax-Span-423">
             <span class="mrow" id="MathJax-Span-424">
              <span class="mi" id="MathJax-Span-425">
               n
              </span>
              <span class="mo" id="MathJax-Span-426">
               −
              </span>
              <span class="mn" id="MathJax-Span-427">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-428">
            )
           </span>
           <span class="mo" id="MathJax-Span-429">
            =
           </span>
           <span class="mi" id="MathJax-Span-430">
            p
           </span>
           <span class="mo" id="MathJax-Span-431">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-432">
            <span class="mi" id="MathJax-Span-433">
             w
            </span>
            <span class="mn" id="MathJax-Span-434">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-435">
            )
           </span>
           <span class="mi" id="MathJax-Span-436">
            p
           </span>
           <span class="mo" id="MathJax-Span-437">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-438">
            <span class="mi" id="MathJax-Span-439">
             w
            </span>
            <span class="mn" id="MathJax-Span-440">
             2
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-441">
            <span class="mrow" id="MathJax-Span-442">
             <span class="mo" id="MathJax-Span-443">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-444">
            <span class="mi" id="MathJax-Span-445">
             w
            </span>
            <span class="mn" id="MathJax-Span-446">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-447">
            )
           </span>
           <span class="mi" id="MathJax-Span-448">
            p
           </span>
           <span class="mo" id="MathJax-Span-449">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-450">
            <span class="mi" id="MathJax-Span-451">
             w
            </span>
            <span class="mn" id="MathJax-Span-452">
             3
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-453">
            <span class="mrow" id="MathJax-Span-454">
             <span class="mo" id="MathJax-Span-455">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-456">
            <span class="mi" id="MathJax-Span-457">
             w
            </span>
            <span class="mn" id="MathJax-Span-458">
             1
            </span>
           </span>
           <span class="mo" id="MathJax-Span-459">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-460">
            <span class="mi" id="MathJax-Span-461">
             w
            </span>
            <span class="mn" id="MathJax-Span-462">
             2
            </span>
           </span>
           <span class="mo" id="MathJax-Span-463">
            )
           </span>
           <span class="mo" id="MathJax-Span-464">
            …
           </span>
           <span class="mi" id="MathJax-Span-465">
            p
           </span>
           <span class="mo" id="MathJax-Span-466">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-467">
            <span class="mi" id="MathJax-Span-468">
             w
            </span>
            <span class="mi" id="MathJax-Span-469">
             n
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-470">
            <span class="mrow" id="MathJax-Span-471">
             <span class="mo" id="MathJax-Span-472">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-473">
            <span class="mi" id="MathJax-Span-474">
             w
            </span>
            <span class="texatom" id="MathJax-Span-475">
             <span class="mrow" id="MathJax-Span-476">
              <span class="mi" id="MathJax-Span-477">
               n
              </span>
              <span class="mo" id="MathJax-Span-478">
               −
              </span>
              <span class="mn" id="MathJax-Span-479">
               2
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-480">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-481">
            <span class="mi" id="MathJax-Span-482">
             w
            </span>
            <span class="texatom" id="MathJax-Span-483">
             <span class="mrow" id="MathJax-Span-484">
              <span class="mi" id="MathJax-Span-485">
               n
              </span>
              <span class="mo" id="MathJax-Span-486">
               −
              </span>
              <span class="mn" id="MathJax-Span-487">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-488">
            )
           </span>
          </span>
         </span>
        </span>
       </p>
       <p>
        那么，我们在面临实际问题时，如何选择依赖词的个数，即 N 呢？
       </p>
       <ul>
        <li>
         更大的 n：对下一个词出现的约束信息更多，具有更大的
         <strong>
          辨别力
         </strong>
         ；
        </li>
        <li>
         更小的 n：在训练语料库中出现的次数更多，具有更可靠的统计信息，具有更高的
         <strong>
          可靠性。
         </strong>
        </li>
       </ul>
       <p>
        理论上，n 越大越好，经验上，trigram 用的最多，尽管如此，原则上，
        <strong>
         能用 bigram 解决，绝不使用 trigram。
        </strong>
       </p>
       <p>
        从本质上说，这种统计语言模型描述的是有限状态的正则文法语言，而自然语言是不确定性语言，因此与真实语言差异较大，表达能力有限，尤其无法较好的处理长距离依赖语言现象。但是，其抓住了自然语言中的局部性约束（Local Constrain）性质，因此该模型在实际应用中取得了较大的成功。
       </p>
       <h4>
        语言模型训练
       </h4>
       <p>
        通常，通过计算最大似然估计（Maximum Likelihood Estimate）构造语言模型，这是对训练数据的最佳估计，如 bigram 公式如下：
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-10-Frame">
         <span class="math" id="MathJax-Span-489">
          <span class="mrow" id="MathJax-Span-490">
           <span class="mi" id="MathJax-Span-491">
            p
           </span>
           <span class="mo" id="MathJax-Span-492">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-493">
            <span class="mi" id="MathJax-Span-494">
             w
            </span>
            <span class="mi" id="MathJax-Span-495">
             i
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-496">
            <span class="mrow" id="MathJax-Span-497">
             <span class="mo" id="MathJax-Span-498">
              |
             </span>
            </span>
           </span>
           <span class="msubsup" id="MathJax-Span-499">
            <span class="mi" id="MathJax-Span-500">
             w
            </span>
            <span class="texatom" id="MathJax-Span-501">
             <span class="mrow" id="MathJax-Span-502">
              <span class="mi" id="MathJax-Span-503">
               i
              </span>
              <span class="mo" id="MathJax-Span-504">
               −
              </span>
              <span class="mn" id="MathJax-Span-505">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-506">
            )
           </span>
           <span class="mo" id="MathJax-Span-507">
            =
           </span>
           <span class="mi" id="MathJax-Span-508">
            f
           </span>
           <span class="mi" id="MathJax-Span-509">
            r
           </span>
           <span class="mi" id="MathJax-Span-510">
            a
           </span>
           <span class="mi" id="MathJax-Span-511">
            c
           </span>
           <span class="texatom" id="MathJax-Span-512">
            <span class="mrow" id="MathJax-Span-513">
             <span class="mi" id="MathJax-Span-514">
              c
             </span>
             <span class="mi" id="MathJax-Span-515">
              o
             </span>
             <span class="mi" id="MathJax-Span-516">
              u
             </span>
             <span class="mi" id="MathJax-Span-517">
              n
             </span>
             <span class="mi" id="MathJax-Span-518">
              t
             </span>
             <span class="mo" id="MathJax-Span-519">
              (
             </span>
             <span class="msubsup" id="MathJax-Span-520">
              <span class="mi" id="MathJax-Span-521">
               w
              </span>
              <span class="texatom" id="MathJax-Span-522">
               <span class="mrow" id="MathJax-Span-523">
                <span class="mi" id="MathJax-Span-524">
                 i
                </span>
                <span class="mo" id="MathJax-Span-525">
                 −
                </span>
                <span class="mn" id="MathJax-Span-526">
                 1
                </span>
               </span>
              </span>
             </span>
             <span class="mo" id="MathJax-Span-527">
              ,
             </span>
             <span class="msubsup" id="MathJax-Span-528">
              <span class="mi" id="MathJax-Span-529">
               w
              </span>
              <span class="mi" id="MathJax-Span-530">
               i
              </span>
             </span>
             <span class="mo" id="MathJax-Span-531">
              )
             </span>
            </span>
           </span>
           <span class="texatom" id="MathJax-Span-532">
            <span class="mrow" id="MathJax-Span-533">
             <span class="mi" id="MathJax-Span-534">
              c
             </span>
             <span class="mi" id="MathJax-Span-535">
              o
             </span>
             <span class="mi" id="MathJax-Span-536">
              u
             </span>
             <span class="mi" id="MathJax-Span-537">
              n
             </span>
             <span class="mi" id="MathJax-Span-538">
              t
             </span>
             <span class="mo" id="MathJax-Span-539">
              (
             </span>
             <span class="msubsup" id="MathJax-Span-540">
              <span class="mi" id="MathJax-Span-541">
               w
              </span>
              <span class="texatom" id="MathJax-Span-542">
               <span class="mrow" id="MathJax-Span-543">
                <span class="mi" id="MathJax-Span-544">
                 i
                </span>
                <span class="mo" id="MathJax-Span-545">
                 −
                </span>
                <span class="mn" id="MathJax-Span-546">
                 1
                </span>
               </span>
              </span>
             </span>
             <span class="mo" id="MathJax-Span-547">
              )
             </span>
            </span>
           </span>
          </span>
         </span>
        </span>
       </p>
       <p>
        如给定句子集“&lt;s&gt; I am Sam &lt;/s&gt;
       </p>
       <p>
        &lt;s&gt; Sam I am &lt;/s&gt;
       </p>
       <p>
        &lt;s&gt; I do not like green eggs and ham &lt;/s&gt;”
       </p>
       <p>
        部分 bigram 语言模型如下所示：
       </p>
       <p>
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-11-Frame">
         <span class="math" id="MathJax-Span-548">
          <span class="mrow" id="MathJax-Span-549">
           <span class="mi" id="MathJax-Span-550">
            c
           </span>
           <span class="mi" id="MathJax-Span-551">
            o
           </span>
           <span class="mi" id="MathJax-Span-552">
            u
           </span>
           <span class="mi" id="MathJax-Span-553">
            n
           </span>
           <span class="mi" id="MathJax-Span-554">
            t
           </span>
           <span class="mo" id="MathJax-Span-555">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-556">
            <span class="mi" id="MathJax-Span-557">
             w
            </span>
            <span class="mi" id="MathJax-Span-558">
             i
            </span>
           </span>
           <span class="mo" id="MathJax-Span-559">
            )
           </span>
          </span>
         </span>
        </span>
        如下:
       </p>
       <p>
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-12-Frame">
         <span class="math" id="MathJax-Span-560">
          <span class="mrow" id="MathJax-Span-561">
           <span class="mi" id="MathJax-Span-562">
            c
           </span>
           <span class="mi" id="MathJax-Span-563">
            o
           </span>
           <span class="mi" id="MathJax-Span-564">
            u
           </span>
           <span class="mi" id="MathJax-Span-565">
            n
           </span>
           <span class="mi" id="MathJax-Span-566">
            t
           </span>
           <span class="mo" id="MathJax-Span-567">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-568">
            <span class="mi" id="MathJax-Span-569">
             w
            </span>
            <span class="texatom" id="MathJax-Span-570">
             <span class="mrow" id="MathJax-Span-571">
              <span class="mi" id="MathJax-Span-572">
               i
              </span>
              <span class="mo" id="MathJax-Span-573">
               −
              </span>
              <span class="mn" id="MathJax-Span-574">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-575">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-576">
            <span class="mi" id="MathJax-Span-577">
             w
            </span>
            <span class="mi" id="MathJax-Span-578">
             i
            </span>
           </span>
           <span class="mo" id="MathJax-Span-579">
            )
           </span>
          </span>
         </span>
        </span>
        如下:
       </p>
       <p>
       </p>
       <p>
        则 bigram 为：
       </p>
       <p>
       </p>
       <p>
        那么，句子“
        <strong>
         &lt;s&gt; I want chinese food &lt;/s&gt;
        </strong>
        ”的概率为：
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-13-Frame">
         <span class="math" id="MathJax-Span-580">
          <span class="mrow" id="MathJax-Span-581">
           <span class="mi" id="MathJax-Span-582">
            p
           </span>
           <span class="mo" id="MathJax-Span-583">
            (
           </span>
           <span class="mo" id="MathJax-Span-584">
            &lt;
           </span>
           <span class="mi" id="MathJax-Span-585">
            s
           </span>
           <span class="mo" id="MathJax-Span-586">
            &gt;
           </span>
           <span class="mi" id="MathJax-Span-587">
            I
           </span>
           <span class="mi" id="MathJax-Span-588">
            w
           </span>
           <span class="mi" id="MathJax-Span-589">
            a
           </span>
           <span class="mi" id="MathJax-Span-590">
            n
           </span>
           <span class="mi" id="MathJax-Span-591">
            t
           </span>
           <span class="mi" id="MathJax-Span-592">
            c
           </span>
           <span class="mi" id="MathJax-Span-593">
            h
           </span>
           <span class="mi" id="MathJax-Span-594">
            i
           </span>
           <span class="mi" id="MathJax-Span-595">
            n
           </span>
           <span class="mi" id="MathJax-Span-596">
            e
           </span>
           <span class="mi" id="MathJax-Span-597">
            s
           </span>
           <span class="mi" id="MathJax-Span-598">
            e
           </span>
           <span class="mi" id="MathJax-Span-599">
            f
           </span>
           <span class="mi" id="MathJax-Span-600">
            o
           </span>
           <span class="mi" id="MathJax-Span-601">
            o
           </span>
           <span class="mi" id="MathJax-Span-602">
            d
           </span>
           <span class="mo" id="MathJax-Span-603">
            &lt;
           </span>
           <span class="texatom" id="MathJax-Span-604">
            <span class="mrow" id="MathJax-Span-605">
             <span class="mo" id="MathJax-Span-606">
              /
             </span>
            </span>
           </span>
           <span class="mi" id="MathJax-Span-607">
            s
           </span>
           <span class="mo" id="MathJax-Span-608">
            &gt;
           </span>
           <span class="mo" id="MathJax-Span-609">
            )
           </span>
           <span class="texatom" id="MathJax-Span-610">
            <span class="mrow" id="MathJax-Span-611">
             <span class="mo" id="MathJax-Span-612">
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-613">
           </span>
           <span class="mo" id="MathJax-Span-614">
            =
           </span>
           <span class="mi" id="MathJax-Span-615">
            p
           </span>
           <span class="mo" id="MathJax-Span-616">
            (
           </span>
           <span class="mi" id="MathJax-Span-617">
            I
           </span>
           <span class="texatom" id="MathJax-Span-618">
            <span class="mrow" id="MathJax-Span-619">
             <span class="mo" id="MathJax-Span-620">
              |
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-621">
            &lt;
           </span>
           <span class="mi" id="MathJax-Span-622">
            s
           </span>
           <span class="mo" id="MathJax-Span-623">
            &gt;
           </span>
           <span class="mo" id="MathJax-Span-624">
            )
           </span>
           <span class="mi" id="MathJax-Span-625">
            P
           </span>
           <span class="mo" id="MathJax-Span-626">
            (
           </span>
           <span class="mi" id="MathJax-Span-627">
            w
           </span>
           <span class="mi" id="MathJax-Span-628">
            a
           </span>
           <span class="mi" id="MathJax-Span-629">
            n
           </span>
           <span class="mi" id="MathJax-Span-630">
            t
           </span>
           <span class="texatom" id="MathJax-Span-631">
            <span class="mrow" id="MathJax-Span-632">
             <span class="mo" id="MathJax-Span-633">
              |
             </span>
            </span>
           </span>
           <span class="mi" id="MathJax-Span-634">
            I
           </span>
           <span class="mo" id="MathJax-Span-635">
            )
           </span>
           <span class="mi" id="MathJax-Span-636">
            p
           </span>
           <span class="mo" id="MathJax-Span-637">
            (
           </span>
           <span class="mi" id="MathJax-Span-638">
            c
           </span>
           <span class="mi" id="MathJax-Span-639">
            h
           </span>
           <span class="mi" id="MathJax-Span-640">
            i
           </span>
           <span class="mi" id="MathJax-Span-641">
            n
           </span>
           <span class="mi" id="MathJax-Span-642">
            e
           </span>
           <span class="mi" id="MathJax-Span-643">
            s
           </span>
           <span class="mi" id="MathJax-Span-644">
            e
           </span>
           <span class="texatom" id="MathJax-Span-645">
            <span class="mrow" id="MathJax-Span-646">
             <span class="mo" id="MathJax-Span-647">
              |
             </span>
            </span>
           </span>
           <span class="mi" id="MathJax-Span-648">
            w
           </span>
           <span class="mi" id="MathJax-Span-649">
            a
           </span>
           <span class="mi" id="MathJax-Span-650">
            n
           </span>
           <span class="mi" id="MathJax-Span-651">
            t
           </span>
           <span class="mo" id="MathJax-Span-652">
            )
           </span>
           <span class="mi" id="MathJax-Span-653">
            p
           </span>
           <span class="mo" id="MathJax-Span-654">
            (
           </span>
           <span class="mi" id="MathJax-Span-655">
            f
           </span>
           <span class="mi" id="MathJax-Span-656">
            o
           </span>
           <span class="mi" id="MathJax-Span-657">
            o
           </span>
           <span class="mi" id="MathJax-Span-658">
            d
           </span>
           <span class="texatom" id="MathJax-Span-659">
            <span class="mrow" id="MathJax-Span-660">
             <span class="mo" id="MathJax-Span-661">
              |
             </span>
            </span>
           </span>
           <span class="mi" id="MathJax-Span-662">
            c
           </span>
           <span class="mi" id="MathJax-Span-663">
            h
           </span>
           <span class="mi" id="MathJax-Span-664">
            i
           </span>
           <span class="mi" id="MathJax-Span-665">
            n
           </span>
           <span class="mi" id="MathJax-Span-666">
            e
           </span>
           <span class="mi" id="MathJax-Span-667">
            s
           </span>
           <span class="mi" id="MathJax-Span-668">
            e
           </span>
           <span class="mo" id="MathJax-Span-669">
            )
           </span>
           <span class="mi" id="MathJax-Span-670">
            p
           </span>
           <span class="mo" id="MathJax-Span-671">
            (
           </span>
           <span class="mo" id="MathJax-Span-672">
            &lt;
           </span>
           <span class="texatom" id="MathJax-Span-673">
            <span class="mrow" id="MathJax-Span-674">
             <span class="mo" id="MathJax-Span-675">
              /
             </span>
            </span>
           </span>
           <span class="mi" id="MathJax-Span-676">
            s
           </span>
           <span class="mo" id="MathJax-Span-677">
            &gt;
           </span>
           <span class="texatom" id="MathJax-Span-678">
            <span class="mrow" id="MathJax-Span-679">
             <span class="mo" id="MathJax-Span-680">
              |
             </span>
            </span>
           </span>
           <span class="mi" id="MathJax-Span-681">
            f
           </span>
           <span class="mi" id="MathJax-Span-682">
            o
           </span>
           <span class="mi" id="MathJax-Span-683">
            o
           </span>
           <span class="mi" id="MathJax-Span-684">
            d
           </span>
           <span class="mo" id="MathJax-Span-685">
            )
           </span>
           <span class="mo" id="MathJax-Span-686">
           </span>
           <span class="mo" id="MathJax-Span-687">
            =
           </span>
           <span class="mn" id="MathJax-Span-688">
            .000031
           </span>
          </span>
         </span>
        </span>
       </p>
       <p>
        为了避免数据溢出、提高性能，通常会使用取 log 后使用加法运算替代乘法运算。
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-14-Frame">
         <span class="math" id="MathJax-Span-689">
          <span class="mrow" id="MathJax-Span-690">
           <span class="mi" id="MathJax-Span-691">
            l
           </span>
           <span class="mi" id="MathJax-Span-692">
            o
           </span>
           <span class="mi" id="MathJax-Span-693">
            g
           </span>
           <span class="mo" id="MathJax-Span-694">
            (
           </span>
           <span class="mi" id="MathJax-Span-695">
            p
           </span>
           <span class="mn" id="MathJax-Span-696">
            1
           </span>
           <span class="mo" id="MathJax-Span-697">
            ∗
           </span>
           <span class="mi" id="MathJax-Span-698">
            p
           </span>
           <span class="mn" id="MathJax-Span-699">
            2
           </span>
           <span class="mo" id="MathJax-Span-700">
            ∗
           </span>
           <span class="mi" id="MathJax-Span-701">
            p
           </span>
           <span class="mn" id="MathJax-Span-702">
            3
           </span>
           <span class="mo" id="MathJax-Span-703">
            ∗
           </span>
           <span class="mi" id="MathJax-Span-704">
            p
           </span>
           <span class="mn" id="MathJax-Span-705">
            4
           </span>
           <span class="mo" id="MathJax-Span-706">
            )
           </span>
           <span class="mo" id="MathJax-Span-707">
            =
           </span>
           <span class="mi" id="MathJax-Span-708">
            l
           </span>
           <span class="mi" id="MathJax-Span-709">
            o
           </span>
           <span class="mi" id="MathJax-Span-710">
            g
           </span>
           <span class="mo" id="MathJax-Span-711">
            (
           </span>
           <span class="mi" id="MathJax-Span-712">
            p
           </span>
           <span class="mn" id="MathJax-Span-713">
            1
           </span>
           <span class="mo" id="MathJax-Span-714">
            )
           </span>
           <span class="mo" id="MathJax-Span-715">
            +
           </span>
           <span class="mi" id="MathJax-Span-716">
            l
           </span>
           <span class="mi" id="MathJax-Span-717">
            o
           </span>
           <span class="mi" id="MathJax-Span-718">
            g
           </span>
           <span class="mo" id="MathJax-Span-719">
            (
           </span>
           <span class="mi" id="MathJax-Span-720">
            p
           </span>
           <span class="mn" id="MathJax-Span-721">
            2
           </span>
           <span class="mo" id="MathJax-Span-722">
            )
           </span>
           <span class="mo" id="MathJax-Span-723">
            +
           </span>
           <span class="mi" id="MathJax-Span-724">
            l
           </span>
           <span class="mi" id="MathJax-Span-725">
            o
           </span>
           <span class="mi" id="MathJax-Span-726">
            g
           </span>
           <span class="mo" id="MathJax-Span-727">
            (
           </span>
           <span class="mi" id="MathJax-Span-728">
            p
           </span>
           <span class="mn" id="MathJax-Span-729">
            3
           </span>
           <span class="mo" id="MathJax-Span-730">
            )
           </span>
           <span class="mo" id="MathJax-Span-731">
            +
           </span>
           <span class="mi" id="MathJax-Span-732">
            l
           </span>
           <span class="mi" id="MathJax-Span-733">
            o
           </span>
           <span class="mi" id="MathJax-Span-734">
            g
           </span>
           <span class="mo" id="MathJax-Span-735">
            (
           </span>
           <span class="mi" id="MathJax-Span-736">
            p
           </span>
           <span class="mn" id="MathJax-Span-737">
            4
           </span>
           <span class="mo" id="MathJax-Span-738">
            )
           </span>
          </span>
         </span>
        </span>
       </p>
       <p>
        网上有一些公开的 n-gram 数据集：
       </p>
       <ul>
        <li>
         Google Web1T5-gram（
         <a href="http://googleresearch.blogspot.com/2006/08/all-our-n-gram-are-belong-to-you.html" rel="nofollow" target="_blank">
          http://googleresearch.blogspot.com/2006/08/all-our-n-gram-are-belong-to-you.html
         </a>
         ）
        </li>
       </ul>
       <p>
        Total number of tokens: 1,306,807,412,486
       </p>
       <p>
        Total number of sentences: 150,727,365,731
       </p>
       <p>
        Total number of unigrams: 95,998,281
       </p>
       <p>
        Total number of bigrams: 646,439,858
       </p>
       <p>
        Total number of trigrams: 1,312,972,925
       </p>
       <p>
        Total number of fourgrams: 1,396,154,236
       </p>
       <p>
        Total number of fivegrams: 1,149,361,413
       </p>
       <p>
        Total number of n-grams: 4,600,926,713
       </p>
       <ul>
        <li>
         Google Book N-grams（
         <a href="http://books.google.com/ngrams/" rel="nofollow" target="_blank">
          http://books.google.com/ngrams/
         </a>
         ）
        </li>
        <li>
         Chinese Web 5-gram（
         <a href="http://www.ldc.upenn.edu/Catalog/catalogEntry.jsp?catalogId=LDC2010T06">
          http://www.ldc.upenn.edu/Catalog/catalogEntry.jsp?catalogId=LDC2010T06
         </a>
         ）
        </li>
       </ul>
       <h4>
        语言模型效果评估
       </h4>
       <p>
        语言模型构造完成后，如何确定其好坏呢？目前主要有两种评价方法：
       </p>
       <ul>
        <li>
         实用方法：通过查看该模型在实际应用（如拼写检查、机器翻译）中的表现来评价，优点是直观、实用，缺点是缺乏针对性、不够客观；
        </li>
        <li>
         理论方法：迷惑度/困惑度/混乱度（preplexity），其基本思想是给测试集赋予较高概率值的语言模型较好，公式如下：
        </li>
       </ul>
       <p>
       </p>
       <p>
        由公式可知，迷惑度越小，句子概率越大，语言模型越好。使用《华尔街日报》，训练数据规模为38 million words，构造 n-gram 语言模型，测试集规模为 1.5 million words，迷惑度如下表所示：
       </p>
       <p>
       </p>
       <h4>
        数据稀疏与平滑技术
       </h4>
       <p>
        大规模数据统计方法与有限的训练语料之间必然产生数据稀疏问题，导致零概率问题，符合经典的 zip’f 定律。如 IBM Brown：366M 英语语料训练 trigram，在测试语料中，有14.7% 的 trigram 和 2.2% 的bigram 在训练语料中未出现。
       </p>
       <p>
        数据稀疏问题定义：“The problem of data sparseness, also known as the zero-frequency problem arises when analyses contain configurations that never occurred in the training corpus.  Then it is not possible to estimate probabilities from observed frequencies, and some other estimation scheme that can generalize (that configurations) from the training data has to be used. —— Dagan”。
       </p>
       <p>
        为了解决数据稀疏问题，人们为理论模型实用化而进行了众多尝试与努力，诞生了一系列经典的平滑技术，它们的基本思想是“降低已出现 n-gram 的条件概率分布，以使未出现的 n-gram 条件概率分布非零”，且经数据平滑后一定保证概率和为1，数据平滑前后如下图所示：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/smoothing.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/smoothing.jpg"/>
        </a>
       </p>
       <ul>
        <li>
         <strong>
          Add-one（Laplace） Smoothing
         </strong>
        </li>
       </ul>
       <p>
        加一平滑法，又称拉普拉斯定律，其保证每个 n-gram 在训练语料中至少出现 1次，以 bigram 为例，公式如下：
       </p>
       <p>
       </p>
       <p>
        其中，V 是所有 bigram 的个数。
       </p>
       <p>
        承接上一节给的例子，经 Add-one Smoothing 后，
        <span class="MathJax" id="MathJax-Element-15-Frame">
         <span class="math" id="MathJax-Span-739">
          <span class="mrow" id="MathJax-Span-740">
           <span class="mi" id="MathJax-Span-741">
            c
           </span>
           <span class="mo" id="MathJax-Span-742">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-743">
            <span class="mi" id="MathJax-Span-744">
             w
            </span>
            <span class="texatom" id="MathJax-Span-745">
             <span class="mrow" id="MathJax-Span-746">
              <span class="mi" id="MathJax-Span-747">
               i
              </span>
              <span class="mo" id="MathJax-Span-748">
               −
              </span>
              <span class="mn" id="MathJax-Span-749">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-750">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-751">
            <span class="mi" id="MathJax-Span-752">
             w
            </span>
            <span class="mi" id="MathJax-Span-753">
             i
            </span>
           </span>
           <span class="mo" id="MathJax-Span-754">
            )
           </span>
          </span>
         </span>
        </span>
        如下所示：
       </p>
       <p>
       </p>
       <p>
        则 bigram 为：
       </p>
       <p>
       </p>
       <p>
        在
        <span class="MathJax" id="MathJax-Element-16-Frame">
         <span class="math" id="MathJax-Span-755">
          <span class="mrow" id="MathJax-Span-756">
           <span class="mi" id="MathJax-Span-757">
            V
           </span>
           <span class="mo" id="MathJax-Span-758">
            &gt;&gt;
           </span>
           <span class="mi" id="MathJax-Span-759">
            c
           </span>
           <span class="mo" id="MathJax-Span-760">
            (
           </span>
           <span class="msubsup" id="MathJax-Span-761">
            <span class="mi" id="MathJax-Span-762">
             w
            </span>
            <span class="texatom" id="MathJax-Span-763">
             <span class="mrow" id="MathJax-Span-764">
              <span class="mi" id="MathJax-Span-765">
               i
              </span>
              <span class="mo" id="MathJax-Span-766">
               −
              </span>
              <span class="mn" id="MathJax-Span-767">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-768">
            ,
           </span>
           <span class="msubsup" id="MathJax-Span-769">
            <span class="mi" id="MathJax-Span-770">
             w
            </span>
            <span class="mi" id="MathJax-Span-771">
             i
            </span>
           </span>
           <span class="mo" id="MathJax-Span-772">
            )
           </span>
          </span>
         </span>
        </span>
        时，即训练语料库中绝大部分 n-gram 未出现的情况（一般都是如此），Add-one Smoothing后有些“喧宾夺主”的现象，效果不佳。如下图所示：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/add1.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/add1.jpg"/>
        </a>
       </p>
       <p>
        一种简单的改进方法是 add-δ smoothing (Lidstone, 1920; Jeffreys, 1948)。δ 是一个小于 1 的数。
       </p>
       <ul>
        <li>
         <strong>
          Good-Turing Smoothing
         </strong>
        </li>
       </ul>
       <p>
        其基本思想是利用频率的类别信息对频率进行平滑。如下图所示：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/goodturing.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/goodturing.jpg"/>
        </a>
       </p>
       <p>
        其中，
        <span class="MathJax" id="MathJax-Element-17-Frame">
         <span class="math" id="MathJax-Span-773">
          <span class="mrow" id="MathJax-Span-774">
           <span class="msubsup" id="MathJax-Span-775">
            <span class="mi" id="MathJax-Span-776">
             N
            </span>
            <span class="mi" id="MathJax-Span-777">
             c
            </span>
           </span>
          </span>
         </span>
        </span>
        表示出现次数为 c 的 n-gram 的个数。调整出现频率为 c 的 n-gram 折扣频率为
        <span class="MathJax" id="MathJax-Element-18-Frame">
         <span class="math" id="MathJax-Span-778">
          <span class="mrow" id="MathJax-Span-779">
           <span class="msubsup" id="MathJax-Span-780">
            <span class="mi" id="MathJax-Span-781">
             c
            </span>
            <span class="texatom" id="MathJax-Span-782">
             <span class="mrow" id="MathJax-Span-783">
              <span class="mo" id="MathJax-Span-784">
               ∗
              </span>
             </span>
            </span>
           </span>
          </span>
         </span>
        </span>
        ：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/goodturing-4.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/goodturing-4.jpg"/>
        </a>
       </p>
       <p>
        但是，对于较大的 c，可能出现
        <span class="MathJax" id="MathJax-Element-19-Frame">
         <span class="math" id="MathJax-Span-785">
          <span class="mrow" id="MathJax-Span-786">
           <span class="msubsup" id="MathJax-Span-787">
            <span class="mi" id="MathJax-Span-788">
             N
            </span>
            <span class="texatom" id="MathJax-Span-789">
             <span class="mrow" id="MathJax-Span-790">
              <span class="mi" id="MathJax-Span-791">
               c
              </span>
              <span class="mo" id="MathJax-Span-792">
               +
              </span>
              <span class="mn" id="MathJax-Span-793">
               1
              </span>
             </span>
            </span>
           </span>
           <span class="mo" id="MathJax-Span-794">
            =
           </span>
           <span class="mn" id="MathJax-Span-795">
            0
           </span>
          </span>
         </span>
        </span>
        或者
        <span class="MathJax" id="MathJax-Element-20-Frame">
         <span class="math" id="MathJax-Span-796">
          <span class="mrow" id="MathJax-Span-797">
           <span class="msubsup" id="MathJax-Span-798">
            <span class="mi" id="MathJax-Span-799">
             N
            </span>
            <span class="mi" id="MathJax-Span-800">
             c
            </span>
           </span>
           <span class="mo" id="MathJax-Span-801">
            &gt;
           </span>
           <span class="msubsup" id="MathJax-Span-802">
            <span class="mi" id="MathJax-Span-803">
             N
            </span>
            <span class="texatom" id="MathJax-Span-804">
             <span class="mrow" id="MathJax-Span-805">
              <span class="mi" id="MathJax-Span-806">
               c
              </span>
              <span class="mo" id="MathJax-Span-807">
               +
              </span>
              <span class="mn" id="MathJax-Span-808">
               1
              </span>
             </span>
            </span>
           </span>
          </span>
         </span>
        </span>
        的情况，反而使得模型质量变差，如下图所示：
       </p>
       <p>
       </p>
       <p>
        直接的改进策略就是“对出现次数超过某个阈值的 gram 不进行平滑，阈值一般取8~10，其他方法请参见“
        <a href="http://faculty.cs.byu.edu/~ringger/CS479/papers/Gale-SimpleGoodTuring.pdf" rel="nofollow" target="_blank">
         Simple Good-Turing
        </a>
        ”。
       </p>
       <p>
        实际中，很少单独使用 Good Turing，而是将其应用到 backoff 平滑算法中。
       </p>
       <ul>
        <li>
         <strong>
          Interpolation
         </strong>
         <strong>
          Smoothing
         </strong>
        </li>
       </ul>
       <p>
        不管是 Add-one，还是 Good Turing 平滑技术，对于未出现的 n-gram 都一视同仁，难免存在不合理性（事件发生概率存在差别），所以这里再介绍一种线性插值平滑技术，其基本思想是将高阶模型和低阶模型作线性组合，利用低元 n-gram 模型对高元 n-gram 模型进行线性插值。因为在没有足够的数据对高元 n-gram 模型进行概率估计时，低元 n-gram 模型通常可以提供有用的信息。公式如下（下方为上下文相关的扩展表示）：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/interpolation.png">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/interpolation.png"/>
        </a>
       </p>
       <p>
       </p>
       <p>
        <span class="MathJax" id="MathJax-Element-21-Frame">
         <span class="math" id="MathJax-Span-809">
          <span class="mrow" id="MathJax-Span-810">
           <span class="mi" id="MathJax-Span-811">
            l
           </span>
           <span class="mi" id="MathJax-Span-812">
            a
           </span>
           <span class="mi" id="MathJax-Span-813">
            m
           </span>
           <span class="mi" id="MathJax-Span-814">
            b
           </span>
           <span class="mi" id="MathJax-Span-815">
            d
           </span>
           <span class="mi" id="MathJax-Span-816">
            a
           </span>
           <span class="mi" id="MathJax-Span-817">
            s
           </span>
          </span>
         </span>
        </span>
        可以通过 EM 算法来估计，具体步骤如下：
       </p>
       <ul>
        <ul>
         <li>
          首先，确定三种数据：Training data、Held-out data 和Test data；
         </li>
        </ul>
       </ul>
       <p>
       </p>
       <ul>
        <ul>
         <li>
          然后，根据 Training data 构造初始的语言模型，并确定初始的
          <span class="MathJax" id="MathJax-Element-22-Frame">
           <span class="math" id="MathJax-Span-818">
            <span class="mrow" id="MathJax-Span-819">
             <span class="mi" id="MathJax-Span-820">
              l
             </span>
             <span class="mi" id="MathJax-Span-821">
              a
             </span>
             <span class="mi" id="MathJax-Span-822">
              m
             </span>
             <span class="mi" id="MathJax-Span-823">
              b
             </span>
             <span class="mi" id="MathJax-Span-824">
              d
             </span>
             <span class="mi" id="MathJax-Span-825">
              a
             </span>
             <span class="mi" id="MathJax-Span-826">
              s
             </span>
            </span>
           </span>
          </span>
          （如均为平均值，保证和为1）；
         </li>
        </ul>
       </ul>
       <ul>
        <ul>
         <li>
          最后，基于 EM 算法迭代地优化
          <span class="MathJax" id="MathJax-Element-23-Frame">
           <span class="math" id="MathJax-Span-827">
            <span class="mrow" id="MathJax-Span-828">
             <span class="mi" id="MathJax-Span-829">
              l
             </span>
             <span class="mi" id="MathJax-Span-830">
              a
             </span>
             <span class="mi" id="MathJax-Span-831">
              m
             </span>
             <span class="mi" id="MathJax-Span-832">
              b
             </span>
             <span class="mi" id="MathJax-Span-833">
              d
             </span>
             <span class="mi" id="MathJax-Span-834">
              a
             </span>
             <span class="mi" id="MathJax-Span-835">
              s
             </span>
            </span>
           </span>
          </span>
          ，使得 Held-out data 概率最大化，如下式所示：
         </li>
        </ul>
       </ul>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/interpolation-1.png">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/interpolation-1.png"/>
        </a>
       </p>
       <ul>
        <li>
         <strong>
          Katz Backoff
         </strong>
        </li>
       </ul>
       <p>
        其基本思想是如果 n-gram 出现频率为 0，就回退到(n-1)-gram 近似求解，如下式所示：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/katz-backoff.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/katz-backoff.jpg"/>
        </a>
       </p>
       <p>
        其中，
        <span class="MathJax" id="MathJax-Element-24-Frame">
         <span class="math" id="MathJax-Span-836">
          <span class="mrow" id="MathJax-Span-837">
           <span class="msubsup" id="MathJax-Span-838">
            <span class="mi" id="MathJax-Span-839">
             P
            </span>
            <span class="texatom" id="MathJax-Span-840">
             <span class="mrow" id="MathJax-Span-841">
              <span class="mo" id="MathJax-Span-842">
               ∗
              </span>
             </span>
            </span>
           </span>
          </span>
         </span>
        </span>
        为折扣概率，而不直接使用 MLE 的概率，比如使用 Good-Turing，这样才能保证
        <a href="http://dataunion.org/wp-content/uploads/2015/02/katz-backoff-2.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/katz-backoff-2.jpg"/>
        </a>
        ，
        <span class="MathJax" id="MathJax-Element-25-Frame">
         <span class="math" id="MathJax-Span-843">
          <span class="mrow" id="MathJax-Span-844">
           <span class="mi" id="MathJax-Span-845">
            a
           </span>
           <span class="mi" id="MathJax-Span-846">
            l
           </span>
           <span class="mi" id="MathJax-Span-847">
            p
           </span>
           <span class="mi" id="MathJax-Span-848">
            h
           </span>
           <span class="mi" id="MathJax-Span-849">
            a
           </span>
          </span>
         </span>
        </span>
        是归一化因子，求解公式如下：
        <a href="http://www.flickering.cn/wp-content/uploads/2015/02/katz-backoff-4.jpg">
         <br/>
        </a>
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/katz-backoff-3.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/katz-backoff-3.jpg"/>
        </a>
       </p>
       <p>
        其他的平滑算法，还有 Absolute discounting，Kneser-Ney Smoothing，Interpolated Kneser-Ney Smoothing 等，在此不再一一详述。
       </p>
       <ul>
        <li>
         <strong>
          Web-scale LMs
         </strong>
        </li>
       </ul>
       <p>
        2007 年 Google Inc. 的 Brants et al. 提出了针对大规模 n-gram 的平滑技术——“
        <a href="http://acl.ldc.upenn.edu/D/D07/D07-1090.pdf%20">
         Stupid Backoff
        </a>
        ”，取得了非常好的效果（有数据，任性），已成功应用于
        <a href="http://acl.ldc.upenn.edu/D/D07/D07-1090.pdf">
         Google 翻译
        </a>
        、
        <a href="http://dejanseo.com.au/research/google/36732.pdf">
         语音搜索
        </a>
        和
        <a href="http://research.google.com/pubs/pub40491.html">
         自动语音识别
        </a>
        等产品中。其 backoff 策略简单粗暴，甚至不保证 n-gram 的概率意义（用 S 代替 P，仅表示相对大小的 Score），公式如下：
       </p>
       <p>
       </p>
       <p>
        数据平滑技术是构造高鲁棒性语言模型的重要手段，且数据平滑的效果与训练语料库的规模有关。训练语料库规模越小，数据平滑的效果越显著；训练语料库规模越大，数据平滑的效果越不显著，甚至可以忽略不计——锦上添花。
       </p>
       <h4>
        语言模型训练实战
       </h4>
       <p>
        n-gram 语言模型的训练流程如下图所示：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/n-gram-training.gif">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/n-gram-training.gif"/>
        </a>
       </p>
       <p>
        以 MLE 为例，可以将模型训练拆分成三个 MapReduce 任务（在此基础上很容易增加平滑功能），如下表所示：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/n-gram-mr.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/n-gram-mr.jpg"/>
        </a>
       </p>
       <p>
        如果不想自己动手实现，推荐几款开源的语言模型项目：
       </p>
       <ul>
        <li>
         SRILM（
         <a href="http://www.speech.sri.com/projects/srilm/" rel="nofollow" target="_blank">
          http://www.speech.sri.com/projects/srilm/
         </a>
         ）
        </li>
        <li>
         IRSTLM（
         <a href="http://hlt.fbk.eu/en/irstlm" rel="nofollow" target="_blank">
          http://hlt.fbk.eu/en/irstlm
         </a>
         ）
        </li>
        <li>
         MITLM（
         <a href="http://code.google.com/p/mitlm/" rel="nofollow" target="_blank">
          http://code.google.com/p/mitlm/
         </a>
         ）
        </li>
        <li>
         BerkeleyLM（
         <a href="http://code.google.com/p/berkeleylm/" rel="nofollow" target="_blank">
          http://code.google.com/p/berkeleylm/
         </a>
         ）
        </li>
       </ul>
       <p>
        在使用 n-gram 语言模型时，也有一些技巧在里面。例如，面对 Google N-gram 语料库，压缩文件大小为 27.9G，解压后 1T 左右，如此庞大的语料资源，使用前一般需要先剪枝（Pruning）处理，缩小规模，如仅使用出现频率大于 threshold 的 n-gram，过滤高阶的 n-gram（如仅使用 n&lt;=3 的资源），基于熵值剪枝，等等。
       </p>
       <p>
        另外，在存储方面也需要做一些优化，如使用 Trie 数据结构存储，借助 bloom filter 辅助查询，把 string 映射为 int 类型处理（基于 huffman 编码、varint 等方法），float/double 转成 varint 或 int 类型（如概率值精确到小数点后 6 位，然后乘 10E6，即可将浮点数转为整数）。
       </p>
       <h4>
        语言模型变种
       </h4>
       <ul>
        <li>
         <strong>
          Class-based N-gram Model
         </strong>
        </li>
       </ul>
       <p>
        该方法基于词类建立语言模型，以缓解数据稀疏问题，且可以方便融合部分语法信息。
       </p>
       <ul>
        <li>
         <strong>
          Topic-based N-gram Model
         </strong>
        </li>
       </ul>
       <p>
        该方法将训练集按主题划分成多个子集，并对每个子集分别建立N-gram语言模型，以解决语言模型的主题自适应问题。架构如下：
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/topic-based-ngram.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/topic-based-ngram.jpg"/>
        </a>
       </p>
       <ul>
        <li>
         <strong>
          Cache-based N-gram Model
         </strong>
        </li>
       </ul>
       <p>
        该方法利用 cache 缓存前一时刻的信息，以用于计算当前时刻概率，以解决语言模型动态自适应问题。这源于：
       </p>
       <p>
        -People tends to use words as few as possible in the article.
       </p>
       <p>
        -If a word has been used, it would possibly be used again in the future.
       </p>
       <p>
        架构如下图所示：
       </p>
       <p>
        <img src="http://dataunion.org/wp-content/uploads/2015/02/cache-based-ngram.jpg"/>
       </p>
       <p>
        像 QQ、搜狗、谷歌等智能拼音输入法就可以采用此策略（声明：作者并不清楚实际产品中的设计），即针对用户个性化输入日志建立基于 cache 的语言模型，用于对通用语言模型输出结果的调权，还可以丰富个性化词条，实现输入法的个性化、智能化。由于动态自适应模块的引入，产品越用越智能，越用越好用，越用越上瘾。
       </p>
       <ul>
        <li>
         <strong>
          Skipping N-gram Model &amp;
         </strong>
         <strong>
          Trigger-based N-gram Model
         </strong>
        </li>
       </ul>
       <p>
        二者核心思想都是刻画远距离约束关系。
       </p>
       <ul>
        <li>
         <strong>
          指数语言模型：最大熵模型 MaxEnt、最大熵马尔科夫模型 MEMM、条件随机域模型 CRF
         </strong>
        </li>
       </ul>
       <p>
        传统的 n-gram 语言模型，只是考虑了词形方面的特征，而没有词性以及语义层面上的知识，并且数据稀疏问题严重，经典的平滑技术也都是从统计学角度解决，未考虑语法、语义等语言学作用。
       </p>
       <p>
        MaxEnt、MEMM、CRF 可以更好的融入多种知识源，刻画语言序列特点，常用于解决序列标注问题。如下图所示，对比了 HMM、MEMM 和 CRF 结构。
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/hmm-memm-crf.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/hmm-memm-crf.jpg"/>
        </a>
       </p>
       <ul>
        <li>
         <strong>
          神经网络语言模型（Neural Network Language Model）
         </strong>
        </li>
       </ul>
       <p>
        神经网络语言模型最早由 Bengio 提出，并进行了系统的研究[Bengio, 2000, 2003]。 随着深度学习的发展，神经网络相关研究越来越深入，神经网络语言模型受到学术界和工业界越来越多的关注，成为了自然语言处理领域最热门的话题之一。神经网络语言模型可以看做一种特殊的模型平滑方式。
       </p>
       <p>
        特别的，在神经网络语言模型有一个副产品—词向量（Word Embedding），应用非常广泛。词向量源于 Hinton 在
        <a href="http://www.cogsci.ucsd.edu/~ajyu/Teaching/Cogs202_sp12/Readings/hinton86.pdf">
         Learning distributed representations of concepts
        </a>
        提出的 Distributed Representation，Bengio[Bengio, 2003] 将其引入到语言模型建模中，提出了神经网络语言模型。不过，目前有很多研究工作是在神经网络语言模型架构下，主要目的就是获取Word、Phrase、Document、Sense、Knowledge 等属性的向量表示，以解决特定的应用任务，如
        <a href="http://ir.hit.edu.cn/~dytang/paper/sswe/14ACL.pdf">
         情感分析
        </a>
        ，
        <a href="http://arxiv.org/abs/1502.04163">
         推荐系统
        </a>
        等。
       </p>
       <p>
        <a href="http://dataunion.org/wp-content/uploads/2015/02/nn-lm.jpg">
         <img src="http://dataunion.org/wp-content/uploads/2015/02/nn-lm.jpg"/>
        </a>
       </p>
       <h4>
        相关推荐
       </h4>
       <ul>
        <li>
         Daniel Jurafskyand James H. Martin. 2008. 《
         <i>
          <a href="http://www.cs.colorado.edu/~martin/SLP/">
           Speech and Language Processing: An Introduction to Natural Language Processing, Computational Linguistics, and Speech Recognition
          </a>
         </i>
         》
         <i>
          .
         </i>
         2nd edition
         <i>
          .
         </i>
        </li>
        <li>
         Chris Manning and andHinrichSchütze. 2002. 《
         <i>
          <a href="http://nlp.stanford.edu/fsnlp/">
           Foundations of Statistical Natural Language Processing
          </a>
         </i>
         》
         <i>
          .
         </i>
        </li>
        <li>
         Dan Jurafsky and Chris Manning. 2012. Stanford University Open Course:
         <i>
          <a href="https://class.coursera.org/nlp/lecture/preview/index">
           Natural Language Processing
          </a>
         </i>
         . Coursera.
        </li>
        <li>
         Stanley F. Chen and Joshua Goodman. 1998.
         <i>
          <a href="http://research.microsoft.com/en-us/um/people/joshuago/tr-10-98.pdf">
           An
          </a>
         </i>
         <i>
          <a href="http://research.microsoft.com/en-us/um/people/joshuago/tr-10-98.pdf">
           Empirical Study of Smoothing Techniques
          </a>
         </i>
         <i>
          <a href="http://research.microsoft.com/en-us/um/people/joshuago/tr-10-98.pdf">
           for Language Modeling
          </a>
         </i>
         <a href="http://research.microsoft.com/en-us/um/people/joshuago/tr-10-98.pdf">
          .
         </a>
        </li>
        <li>
         关毅. 2007.  哈工大：
         <i>
          <a href="http://wenku.baidu.com/search?word=%CD%B3%BC%C6%D7%D4%C8%BB%D3%EF%D1%D4%B4%A6%C0%ED+%B9%D8%D2%E3&amp;lm=3&amp;od=0&amp;fr=top_search">
           统计自然语言处理
          </a>
         </i>
         .
        </li>
        <li>
         肖镜辉, 2007,
         <i>
          <a href="http://www.insun.hit.edu.cn/upfiles/2007-5-30%20%E8%82%96%E9%95%9C%E8%BE%89%20%E7%BB%9F%E8%AE%A1%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B%E7%AE%80%E4%BB%8B.pdf">
           统计
          </a>
         </i>
         <i>
          <a href="http://www.insun.hit.edu.cn/upfiles/2007-5-30%20%E8%82%96%E9%95%9C%E8%BE%89%20%E7%BB%9F%E8%AE%A1%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B%E7%AE%80%E4%BB%8B.pdf">
           语言模型
          </a>
         </i>
         <i>
          <a href="http://www.insun.hit.edu.cn/upfiles/2007-5-30%20%E8%82%96%E9%95%9C%E8%BE%89%20%E7%BB%9F%E8%AE%A1%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B%E7%AE%80%E4%BB%8B.pdf">
           简介
          </a>
         </i>
         .
        </li>
        <li>
         吴军. 2012. 《
         <i>
          <a href="http://www.amazon.cn/%E6%95%B0%E5%AD%A6%E4%B9%8B%E7%BE%8E-%E5%90%B4%E5%86%9B/dp/B0084ASO7E/ref=sr_1_1?ie=UTF8&amp;qid=1351056283&amp;sr=8-1">
           数学之美
          </a>
         </i>
         <i>
          》.
         </i>
        </li>
        <li>
         fandywang, 2012,《
         <a href="http://flickering.qiniudn.com/flickeringNLP&amp;%E7%BB%9F%E8%AE%A1%E8%AF%AD%E8%A8%80%E6%A8%A1%E5%9E%8B(fandywang201211).pdf">
          NLP&amp;统计语言模型
         </a>
         》.
        </li>
        <li>
         Thorsten Brants, Ashok C. Popat, PengXu, Franz J. Och, Jeffrey Dean. ACL 2007.
         <i>
          <a href="http://acl.ldc.upenn.edu/D/D07/D07-1090.pdf">
           Large Language Models in Machine
          </a>
         </i>
         <i>
          <a href="http://acl.ldc.upenn.edu/D/D07/D07-1090.pdf">
           Translation
          </a>
         </i>
         .  Google Inc.
        </li>
        <li>
         CiprianChelba, Dan Bikel, Maria Shugrina, Patrick Nguyen, Shankar Kumar. 2012.
         <i>
          <a href="http://research.google.com/pubs/pub40491.html">
           Large
          </a>
         </i>
         <i>
          <a href="http://research.google.com/pubs/pub40491.html">
           Scale Language Modeling in
          </a>
         </i>
         <i>
          <a href="http://research.google.com/pubs/pub40491.html">
           Automatic Speech Recognition
          </a>
         </i>
         . Google Inc.
        </li>
        <li>
         Yoshua Bengio.
         <a class="external text" href="http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf" rel="nofollow">
          A Neural Probabilistic Language Model
         </a>
         . JMLR(2003).
        </li>
        <li>
         刘知远，2014，《
         <a href="http://nlp.csai.tsinghua.edu.cn/~lzy/talks/representation2015.pptx">
          Representation Learning for Word, Sense, Phrase, Document and Knowledge
         </a>
         》.
        </li>
        <li>
         Duyu Tang, Furu Wei, Nan Yang, Ming Zhou, Ting Liu, Bing Qin. ACL 2014. Learning Sentiment-Specific Word Embedding for Twitter Sentiment Classification. [
         <a href="http://ir.hit.edu.cn/~dytang/paper/sswe/14ACL.pdf">
          pdf
         </a>
         ][
         <a href="http://ir.hit.edu.cn/~dytang/paper/sswe/acl-slides.pdf">
          slides
         </a>
         ].
        </li>
        <li>
         SRILM
         <a href="http://www.speech.sri.com/projects/srilm/">
          http://www.speech.sri.com/projects/srilm
         </a>
         <a href="http://www.speech.sri.com/projects/srilm/">
          /
         </a>
        </li>
        <li>
         KenLM
         <a href="http://kheafield.com/code/kenlm/">
          http://kheafield.com/code/kenlm
         </a>
         <a href="http://kheafield.com/code/kenlm/">
          /
         </a>
        </li>
        <li>
         IRSTLM
         <a href="http://hlt.fbk.eu/en/irstlm">
          http://
         </a>
         <a href="http://hlt.fbk.eu/en/irstlm">
          hlt.fbk.eu/en/irstlm
         </a>
        </li>
        <li>
         berkeleylm
         <a href="http://code.google.com/p/berkeleylm/">
          http://code.google.com/p/berkeleylm
         </a>
         <a href="http://code.google.com/p/berkeleylm/">
          /
         </a>
        </li>
        <li>
         The CMU SLM Toolkit
         <a href="http://www.speech.cs.cmu.edu/SLM_info.html">
          http://
         </a>
         <a href="http://www.speech.cs.cmu.edu/SLM_info.html">
          www.speech.cs.cmu.edu/SLM_info.html
         </a>
        </li>
        <li>
         Moses: statistical machine translation system
         <a href="http://www.statmt.org/moses/">
          http
         </a>
         <a href="http://www.statmt.org/moses/">
          ://www.statmt.org/moses/
         </a>
        </li>
       </ul>
       <div>
       </div>
       <div id="myCopyRight">
        <p>
         本文链接：
         <a href="http://www.flickering.cn/nlp/2015/02/%e6%88%91%e4%bb%ac%e6%98%af%e8%bf%99%e6%a0%b7%e7%90%86%e8%a7%a3%e8%af%ad%e8%a8%80%e7%9a%84-2%e7%bb%9f%e8%ae%a1%e8%af%ad%e8%a8%80%e6%a8%a1%e5%9e%8b/" rel="bookmark" title="本文固定链接 http://www.flickering.cn/nlp/2015/02/%e6%88%91%e4%bb%ac%e6%98%af%e8%bf%99%e6%a0%b7%e7%90%86%e8%a7%a3%e8%af%ad%e8%a8%80%e7%9a%84-2%e7%bb%9f%e8%ae%a1%e8%af%ad%e8%a8%80%e6%a8%a1%e5%9e%8b/">
          [我们是这样理解语言的-2]统计语言模型
         </a>
        </p>
       </div>
      </div>
      <div>
       <strong>
        注：转载文章均来自于公开网络，仅供学习使用，不会用于任何商业用途，如果侵犯到原作者的权益，请您与我们联系删除或者授权事宜，联系邮箱：contact@dataunion.org。转载数盟网站文章请注明原文章作者，否则产生的任何版权纠纷与数盟无关。
       </strong>
      </div>
      <!--content_text-->
      <div class="fenxian">
       <!-- JiaThis Button BEGIN -->
       <div class="jiathis_style_32x32">
        <p class="jiathis_button_weixin">
        </p>
        <p class="jiathis_button_tsina">
        </p>
        <p class="jiathis_button_qzone">
        </p>
        <p class="jiathis_button_cqq">
        </p>
        <p class="jiathis_button_tumblr">
        </p>
        <a class="jiathis jiathis_txt jtico jtico_jiathis" href="http://www.jiathis.com/share" target="_blank">
        </a>
        <p class="jiathis_counter_style">
        </p>
       </div>
       <!-- JiaThis Button END -->
      </div>
     </article>
     <!--content-->
     <!--相关文章-->
     <div class="xianguan">
      <div class="xianguantitle">
       相关文章！
      </div>
      <ul class="pic">
       <li>
        <a href="http://dataunion.org/16951.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/05/t01b6b136b6b044f052-300x233.png"/>
        </a>
        <a class="link" href="http://dataunion.org/16951.html" rel="bookmark" title="基于R语言的ggplot2的大数定律和中心极限定理验证">
         基于R语言的ggplot2的大数定律和中心极限定理验证
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/13403.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/03/QQ截图20150328220348.png"/>
        </a>
        <a class="link" href="http://dataunion.org/13403.html" rel="bookmark" title="支撑统计学的七大支柱！">
         支撑统计学的七大支柱！
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/8795.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/01/t0109d2976b094ddfcf-300x211.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/8795.html" rel="bookmark" title="一个大学统计学教授推荐的统计学书单及书评">
         一个大学统计学教授推荐的统计学书单及书评
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/2903.html">
         <img src="http://dataunion.org/wp-content/uploads/2014/06/23-300x150.png"/>
        </a>
        <a class="link" href="http://dataunion.org/2903.html" rel="bookmark" title="【精】上海交大“数据科学”相关免费在线课程推荐~">
         【精】上海交大“数据科学”相关免费在线课程推荐~
        </a>
       </li>
      </ul>
     </div>
     <!--相关文章-->
     <div class="comment" id="comments">
      <!-- You can start editing here. -->
      <!-- If comments are open, but there are no comments. -->
      <div class="title">
       期待你一针见血的评论，Come on！
      </div>
      <div id="respond">
       <p>
        不用想啦，马上
        <a href="http://dataunion.org/wp-login.php?redirect_to=http%3A%2F%2Fdataunion.org%2F11266.html">
         "登录"
        </a>
        发表自已的想法.
       </p>
      </div>
     </div>
     <!-- .nav-single -->
    </div>
    <!--Container End-->
    <aside id="sitebar">
     <div class="sitebar_list2">
      <div class="wptag">
       <span class="tagtitle">
        热门标签+
       </span>
       <div class="tagg">
        <ul class="menu" id="menu-%e5%8f%8b%e6%83%85%e9%93%be%e6%8e%a5">
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1605" id="menu-item-1605">
          <a href="http://taidizh.com/">
           泰迪智慧
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20884" id="menu-item-20884">
          <a href="http://www.transwarp.cn/">
           星环科技
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3538" id="menu-item-3538">
          <a href="http://datall.org/">
           珈和遥感
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20888" id="menu-item-20888">
          <a href="http://www.chinahadoop.cn/">
           小象学院
          </a>
         </li>
        </ul>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <div class="textwidget">
       <div align="center">
        <a href="http://study.163.com/course/courseMain.htm?courseId=991022" target="_blank">
         <img src="http://dataunion.org/wp-content/uploads/2016/03/dv.jpg"/>
        </a>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       文章分类
      </h4>
      <div class="tagcloud">
       <a class="tag-link-44" href="http://dataunion.org/category/industry/demo" style="font-size: 10.204724409449pt;" title="4个话题">
        Demo展示
       </a>
       <a class="tag-link-31" href="http://dataunion.org/category/experts" style="font-size: 15.826771653543pt;" title="52个话题">
        专家团队
       </a>
       <a class="tag-link-870" href="http://dataunion.org/category/tech/ai" style="font-size: 19.795275590551pt;" title="273个话题">
        人工智能
       </a>
       <a class="tag-link-488" href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f" style="font-size: 8pt;" title="1个话题">
        加入数盟
       </a>
       <a class="tag-link-869" href="http://dataunion.org/category/tech/viz" style="font-size: 17.204724409449pt;" title="93个话题">
        可视化
       </a>
       <a class="tag-link-30" href="http://dataunion.org/category/partners" style="font-size: 10.645669291339pt;" title="5个话题">
        合作伙伴
       </a>
       <a class="tag-link-889" href="http://dataunion.org/category/parterc" style="font-size: 11.582677165354pt;" title="8个话题">
        合作会议
       </a>
       <a class="tag-link-104" href="http://dataunion.org/category/books" style="font-size: 12.96062992126pt;" title="15个话题">
        图书
       </a>
       <a class="tag-link-220" href="http://dataunion.org/category/tech/base" style="font-size: 19.850393700787pt;" title="281个话题">
        基础架构
       </a>
       <a class="tag-link-219" href="http://dataunion.org/category/tech/analysis" style="font-size: 19.409448818898pt;" title="232个话题">
        数据分析
       </a>
       <a class="tag-link-887" href="http://dataunion.org/category/tech/dm" style="font-size: 13.291338582677pt;" title="17个话题">
        数据挖掘
       </a>
       <a class="tag-link-34" href="http://dataunion.org/category/tech" style="font-size: 20.732283464567pt;" title="404个话题">
        文章
       </a>
       <a class="tag-link-1" href="http://dataunion.org/category/uncategorized" style="font-size: 22pt;" title="693个话题">
        未分类
       </a>
       <a class="tag-link-4" href="http://dataunion.org/category/events" style="font-size: 14.503937007874pt;" title="29个话题">
        活动
       </a>
       <a class="tag-link-890" href="http://dataunion.org/category/tech/%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0" style="font-size: 10.204724409449pt;" title="4个话题">
        深度学习
       </a>
       <a class="tag-link-221" href="http://dataunion.org/category/tech/devl" style="font-size: 18.968503937008pt;" title="193个话题">
        编程语言
       </a>
       <a class="tag-link-888" href="http://dataunion.org/category/career" style="font-size: 15.661417322835pt;" title="48个话题">
        职业规划
       </a>
       <a class="tag-link-5" href="http://dataunion.org/category/jobs" style="font-size: 14.11811023622pt;" title="25个话题">
        职位
       </a>
       <a class="tag-link-871" href="http://dataunion.org/category/industry" style="font-size: 15.716535433071pt;" title="49个话题">
        行业
       </a>
       <a class="tag-link-613" href="http://dataunion.org/category/industry/case" style="font-size: 16.984251968504pt;" title="84个话题">
        行业应用
       </a>
       <a class="tag-link-885" href="http://dataunion.org/category/industry/news" style="font-size: 17.425196850394pt;" title="102个话题">
        行业资讯
       </a>
       <a class="tag-link-10" href="http://dataunion.org/category/training" style="font-size: 14.228346456693pt;" title="26个话题">
        课程
       </a>
       <a class="tag-link-16" href="http://dataunion.org/category/sources" style="font-size: 15.661417322835pt;" title="48个话题">
        资源
       </a>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       功能
      </h4>
      <ul>
       <li>
        <a href="http://dataunion.org/wp-login.php?action=register">
         注册
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/wp-login.php">
         登录
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/feed">
         文章
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/comments/feed">
         评论
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="https://cn.wordpress.org/" title="基于WordPress，一个优美、先进的个人信息发布平台。">
         WordPress.org
        </a>
       </li>
      </ul>
     </div>
    </aside>
    <div class="clear">
    </div>
   </div>
   <!--main-->
   ﻿
   <footer id="dibu">
    <div class="about">
     <div class="right">
      <ul class="menu" id="menu-%e5%ba%95%e9%83%a8%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-18024" id="menu-item-18024">
        <a href="http://dataunion.org/category/partners">
         合作伙伴
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20881" id="menu-item-20881">
        <a href="http://dataunion.org/contribute">
         文章投稿
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20872" id="menu-item-20872">
        <a href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f">
         加入数盟
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22441" id="menu-item-22441">
        <a href="http://dataunion.org/f-links">
         友情链接
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20874" id="menu-item-20874">
        <a href="http://dataunion.org/aboutus">
         关于数盟
        </a>
       </li>
      </ul>
      <p class="banquan">
       数盟社区        ，
        做最棒的数据科学社区
      </p>
     </div>
     <div class="left">
      <ul class="bottomlist">
       <li>
        <a href="http://weibo.com/DataScientistUnion  " target="_blank" 　title="">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weibo.png"/>
        </a>
       </li>
       <li>
        <a class="cd-popup-trigger" href="http://dataunion.org/11266.html#0">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weixin.png"/>
        </a>
       </li>
      </ul>
      <div class="cd-popup">
       <div class="cd-popup-container">
        <h1>
         扫描二维码,加微信公众号
        </h1>
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/2014-12-06-1515289049.png"/>
        <a class="cd-popup-close" href="http://dataunion.org/11266.html">
        </a>
       </div>
       <!-- cd-popup-container -->
      </div>
      <!-- cd-popup -->
     </div>
    </div>
    <!--about-->
    <div class="bottom">
     <a href="http://dataunion.org/">
      数盟社区
     </a>
     <a href="http://www.miitbeian.gov.cn/" rel="external nofollow" target="_blank">
      京ICP备14026740号
     </a>
     联系我们：
     <a href="mailto:contact@dataunion.org" target="_blank">
      contact@dataunion.org
     </a>
     <div class="tongji">
     </div>
     <!--bottom-->
     <div class="scroll" id="scroll" style="display:none;">
      ︿
     </div>
    </div>
   </footer>
   <!--dibu-->
  </div>
 </body>
</html>